# 去除空行!!
file = open("d:/data/videodata_seg_re_drop.txt", encoding='utf8')
out = open("d:/data/videodata_seg_re_drop_noempty.txt", encoding='utf8', mode='w')
cnt = 0
content = ''
while True:
    cnt += 1
    if cnt % 10000 == 0:
        print(cnt)
    line = file.readline()
    if not line:
        break
    line = line.rstrip()
    if len(line) != 0:
        out.write(line + '\n')
out.close()

from gensim.models.keyedvectors import KeyedVectors
import numpy as np

model = KeyedVectors.load_word2vec_format("model/gensim64_skipgram_merge" + ".vec")

file = open("d:/data/videodata_seg_re_drop_noempty.txt", encoding='utf8')
out = open("d:/data/doc.vec", encoding='utf8', mode='w')
err = 0
cnt = 0
while True:
    line = file.readline()
    if not line:
        break
    line = line.rstrip()
    words = line.split(" ")
    if len(words) == 0:
        err += 1
        continue

    docvec = np.zeros(model.vector_size)
    iszero = True
    for word in words:
        try:
            vec = np.zeros(model.vector_size)
            vec = model.word_vec(word)
            iszero = False
        except KeyError as e:
            pass
        docvec += vec
    if iszero:
        print("word = " + line)
    docvec /= len(words)
    out.write(line.replace(" ", "") + ',')
    for f in docvec:
        out.write(str(f) + ',')
    out.write('\n')

    if cnt % 10000 == 0:
        print(cnt)
    cnt += 1
